from nltk.tokenize import word_tokenize
import re
import string
import emoticon
from stopwords import StopWords
from englishwords import EnglishWords
from stemming.porter2 import stem
class TweetCleanser:
	"""
	Cleanse a tweet/ text document to remove unneccessary elements,
	for the purpose of sentimental analysis

	Class methods:
	__init__: specify location of files that contain list of English words
			  and stop words
	remove_lax_url
	remove_at_tag
	remove_hash_tag
	remove_non_ascii_chars
	remove_punctuation
	remove_digit
	stem_words
	remove_stop_words
	remove_non_english_words
	remove_blank_spaces
	context_cleanse
	cleanse
	"""
	word = None
	st = None	
	def __init__(self,englishWordFile, stopWordFile):
		self.word = EnglishWords(englishWordFile)
		self.st = StopWords(stopWordFile)
	def remove_lax_url(self,s):
		LAX_URL_REGEX = r"(https?|ftp|file)://[-a-zA-Z0-9+&@#/%?=~_|!:\
		,.;]*[-a-zA-Z0-9+&@#/%=~_|]"
		return re.sub(LAX_URL_REGEX,"", s)
	def remove_at_tag(self, s):
		LAX_AT_TAG_REGEX = r"@\S+\b"
		return re.sub(LAX_AT_TAG_REGEX,"", s)
	def remove_lax_hash_tag(self, s):
		LAX_HASH_TAG_REGEX = r"#\S+\b"
		return re.sub(LAX_HASH_TAG_REGEX,"",s)
	def remove_non_ascii_chars(self,s):
		NON_ASCII_CHARS_REGEX = r"[^\x20-\x7e]"
		return re.sub(NON_ASCII_CHARS_REGEX,"",s)
	def remove_punctuation(self,s):
		return s.translate(string.maketrans("",""), string.punctuation)
	def remove_digit(self,s):
		DIGIT_REGEX = r"(^|\b)(-|\+)?\d+\.?\d*($|\b)"
		return re.sub(DIGIT_REGEX,"", s)
	def stem_words(self,s):
		tokens = s.split(' ')
		stringBuffer = []
		for nextToken in tokens:
			stringBuffer.append(stem(nextToken))
		return ' '.join(stringBuffer)
	def remove_stop_words(self,s):
		st = self.st
		tokens = s.split(' ')
		stringBuffer = []
		for nextToken in tokens:
			if st.isStopWord(nextToken)==False:
				stringBuffer.append(nextToken)
		return ' '.join(stringBuffer)
	def remove_non_english_words(self, s):
		word = self.word
		tokens = word_tokenize(s)
		stringBuffer = []
		for nextToken in tokens:
			if word.isEnglishWord(nextToken)==True:
				stringBuffer.append(nextToken)
		return ' '.join(stringBuffer)
	def remove_blank_spaces(self, s):
		tokens = word_tokenize(s)
		stringBuffer = []
		for nextToken in tokens:
			if nextToken.strip()!="":
				stringBuffer.append(nextToken)
		return ' '.join(stringBuffer)
	# 
	def context_cleanse(self, s):
		"""
		Combine three methods: removeStopWords, removeNonEnglishWords,
		removeBlankSpaces into one function to reduce time waste on 
		tokenize the string 3 times
		Arguments:
		s: input string"""
		st = self.st
		word = self.word
		tokens = word_tokenize(s)
		stringBuffer = []
		for nextToken in tokens:
			if st.isStopWord(nextToken)==False \
			and word.isEnglishWord(nextToken)==True\
			and nextToken.strip()!="":
				stringBuffer.append(nextToken)
		return ' '.join(stringBuffer)
	def cleanse(self,s):
		"""
		Arguments:
		s: input string
		"""
		s = s.lower()
		s = emoticon.normalize(s)
		s = self.stem_words(s)
		s = self.remove_lax_url(s)
		s = self.remove_at_tag(s)
		s = self.remove_lax_hash_tag(s)
		s = self.remove_non_ascii_chars(s)
		s = self.remove_punctuation(s)
		s = self.remove_digit(s)
		s = self.remove_stop_words(s)
		s = self.remove_blank_spaces(s)
		s = self.remove_non_english_words(s)
		s = self.context_cleanse(s)
		return s